#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
验证爬取数据脚本
"""

import sys
import asyncio
import logging
from pathlib import Path
from datetime import datetime

# 添加项目根目录到Python路径
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))

from src.crawler.crawler_engine import CrawlerEngine
from src.data.data_processor import DataProcessor
from src.utils.excel_exporter import ExcelExporter


async def verify_crawl_data():
    """验证爬取数据"""
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s'
    )
    
    logger = logging.getLogger(__name__)
    
    try:
        logger.info("开始验证爬取数据...")
        
        # 创建爬虫引擎
        crawler = CrawlerEngine()
        processor = DataProcessor()
        exporter = ExcelExporter()
        
        # 启动爬虫
        await crawler.start(headless=True)
        
        # 爬取1页数据进行验证
        logger.info("开始爬取深圳餐饮数据（限制1页）...")
        raw_data = await crawler.crawl_shenzhen_catering(max_pages=1)
        
        logger.info(f"原始数据获取完成，共 {len(raw_data)} 条")
        
        if raw_data:
            # 显示前5条原始数据的关键字段
            logger.info("前5条原始数据示例:")
            for i, item in enumerate(raw_data[:5]):
                logger.info(f"第{i+1}条:")
                logger.info(f"  联系人: {item.get('contact_person', 'N/A')}")
                logger.info(f"  业态: {item.get('business_type', 'N/A')}")
                logger.info(f"  面积: {item.get('building_area', 'N/A')}")
                logger.info(f"  租金: {item.get('acceptable_rent', 'N/A')}")
                logger.info(f"  区域: {item.get('area_info', 'N/A')}")
                logger.info(f"  详情URL: {item.get('detail_url', 'N/A')}")
                logger.info("")
            
            # 数据处理
            logger.info("开始数据处理...")
            processed_data = processor.process_crawled_data(raw_data)
            
            logger.info(f"处理后数据: {len(processed_data)} 条")
            
            if processed_data:
                logger.info("前3条处理后数据示例:")
                for i, item in enumerate(processed_data[:3]):
                    logger.info(f"第{i+1}条:")
                    logger.info(f"  城市: {item.get('city', 'N/A')}")
                    logger.info(f"  联系人: {item.get('contact_person', 'N/A')}")
                    logger.info(f"  联系方式: {item.get('contact_info', 'N/A')}")
                    logger.info(f"  业态: {item.get('business_type', 'N/A')}")
                    logger.info(f"  承受租金: {item.get('acceptable_rent', 'N/A')}")
                    logger.info(f"  建筑面积: {item.get('building_area', 'N/A')}")
                    logger.info(f"  发布日期: {item.get('publish_date', 'N/A')}")
                    logger.info(f"  详情页URL: {item.get('detail_url', 'N/A')}")
                    logger.info("")
                
                # 导出到Excel
                logger.info("导出数据到Excel...")
                excel_file = exporter.export_leads_data(processed_data, "深圳餐饮线索验证")
                logger.info(f"Excel文件已保存: {excel_file}")
                
                # 生成质量报告
                quality_report = processor.generate_quality_report(processed_data)
                logger.info("数据质量报告:")
                logger.info(f"  总记录数: {quality_report['total_records']}")
                logger.info(f"  有效记录数: {quality_report['valid_records']}")
                logger.info(f"  质量分数: {quality_report['quality_score']}%")
                
                # 字段完整性
                logger.info("字段完整性:")
                for field, percentage in quality_report['field_completeness'].items():
                    logger.info(f"  {field}: {percentage:.1f}%")
                
                return True
            else:
                logger.error("数据处理后无有效数据")
                return False
        else:
            logger.error("未获取到任何数据")
            return False
            
    except Exception as e:
        logger.error(f"验证失败: {e}")
        return False
        
    finally:
        await crawler.stop()


if __name__ == "__main__":
    success = asyncio.run(verify_crawl_data())
    if success:
        print("✅ 数据验证成功")
    else:
        print("❌ 数据验证失败")
